{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "import preprocessor as p\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ../twitter16/twitter16_text_processing.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import spacy\n",
    "from tqdm import tqdm, tqdm_notebook, tnrange\n",
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self,uid,tid,time_stamp,label):\n",
    "        self.children = {}\n",
    "        self.childrenList = []\n",
    "        self.num_children = 0\n",
    "        self.tid = tid\n",
    "        self.uid = uid\n",
    "        self.label = label\n",
    "        self.time_stamp = time_stamp\n",
    "    \n",
    "    def add_child(self,node):\n",
    "        if node.uid not in self.children:\n",
    "            self.children[node.uid] = node\n",
    "            self.num_children += 1\n",
    "        else:\n",
    "            self.children[node.uid] = node\n",
    "        self.childrenList = list(self.children.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tree:\n",
    "    def __init__(self,root):\n",
    "        self.root = root\n",
    "        self.tweet_id = root.tid\n",
    "        self.uid = root.uid\n",
    "        self.height = 0\n",
    "        self.nodes = 0\n",
    "    \n",
    "    def show(self):\n",
    "        queue = [self.root,0]\n",
    "        \n",
    "        while len(queue) != 0:\n",
    "            toprint = queue.pop(0)\n",
    "            if toprint == 0:\n",
    "                print('\\n')\n",
    "            else:\n",
    "                print(toprint.uid,end=' ')\n",
    "                queue += toprint.children.values()\n",
    "                queue.append(0)\n",
    "                \n",
    "    def insertnode(self,curnode,parent,child):\n",
    "        if curnode.uid == parent.uid:\n",
    "            curnode.add_child(child)\n",
    "            return 1\n",
    "\n",
    "        elif parent.uid in curnode.children:\n",
    "            s = self.insertnode(curnode.children[parent.uid],parent,child)\n",
    "            return 2\n",
    "        else:\n",
    "            for node in curnode.children:\n",
    "                s = self.insertnode(curnode.children[node],parent,child)\n",
    "                if s == 2:\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadPklFileNum(datapath,incSize,fileNum):\n",
    "    \n",
    "    with open(datapath+str(incSize)+'inc_'+str(fileNum)+'.pickle', 'rb') as handle:\n",
    "        twitTrees = pkl.load(handle)\n",
    "    return twitTrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadTreeFilesOfIncrement(datapath,incSize):\n",
    "    twittertrees = {}\n",
    "    \n",
    "    files = [x for x in os.listdir(t15Datapath) if str(incSize)+'inc' in x]\n",
    "    \n",
    "    for file in tqdm(files):\n",
    "        with open(datapath+file,'rb') as handle:\n",
    "            partialTrees = pkl.load(handle)\n",
    "        twittertrees.update(partialTrees)\n",
    "        \n",
    "    return twittertrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "t15Datapath = '/home/nikhil.pinnaparaju/Research/Temporal Tree Encoding/twitter16/pickledTrees/'\n",
    "twitter15_trees = loadPklFileNum(t15Datapath,20,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9/9 [01:22<00:00,  9.19s/it]\n"
     ]
    }
   ],
   "source": [
    "twitter15_trees = loadTreeFilesOfIncrement(t15Datapath,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:00<00:00, 9340.44it/s]\n",
      "  0%|          | 0/33 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n",
      "946\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 33/33 [02:59<00:00,  5.43s/it]\n",
      "100%|██████████| 253378/253378 [01:54<00:00, 2215.32it/s]\n"
     ]
    }
   ],
   "source": [
    "%run ../twitter16/userdata_parser.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 253378/253378 [00:01<00:00, 204408.27it/s]\n"
     ]
    }
   ],
   "source": [
    "for key in tqdm(userVects):\n",
    "    userVects[key] = userVects[key].float()\n",
    "\n",
    "userVects = defaultdict(lambda:torch.tensor([1.1100e+02, 1.5000e+01, 0.0000e+00, 7.9700e+02, 4.7300e+02, 0.0000e+00,\n",
    "        8.3326e+04, 1.0000e+00]),userVects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%run ./textEncoders.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ./temporal_tree_model.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = 'cuda:2'\n",
    "    device = 'cpu'\n",
    "else:\n",
    "    device = 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'false': 0, 'true': 1, 'unverified': 2, 'non-rumor': 3}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelMap = {}\n",
    "labelCount = 0\n",
    "for label in list(twitter15_labels.values()):\n",
    "    if label not in labelMap:\n",
    "        labelMap[label] = labelCount\n",
    "        labelCount += 1\n",
    "labelMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10\n",
    "X = []\n",
    "y = []\n",
    "X_text = []\n",
    "\n",
    "for tid in twitter15_trees:\n",
    "        if tid in twitter15_trees and tid in twitter15_labels:\n",
    "            X.append(tuple((twitter15_trees[tid],twitter15_text[tid])))\n",
    "            y.append(labelMap[twitter15_labels[tid]])\n",
    "            X_text.append(twitter15_text[tid])\n",
    "            \n",
    "x_train,x_test,y_train,y_test = train_test_split(X,y,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class jointModel(nn.Module):\n",
    "    def __init__(self, treeEncoderType, textEncoderType, pretrain, textparams, treeparams, X, y, device):\n",
    "        super(jointModel, self).__init__()\n",
    "        if textEncoderType == 'rnn':\n",
    "            self.textEncoderModel = TextEncoder(textEncoderType,textparams,X,y,device)\n",
    "            textparams['hidden_dim'] = textparams['hidden_dim']*self.textEncoderModel.textEncoder.numDirs\n",
    "            \n",
    "        if textEncoderType == 'bert':\n",
    "            self.textEncoderModel = BertTextEncoder(textEncoderType,{},X,y,device)\n",
    "            textparams['hidden_dim'] = 768\n",
    "        \n",
    "        if textEncoderType == 'attn':\n",
    "            self.textEncoderModel = AttentionTextEncoder(textEncoderType,textparams,X,y,device)\n",
    "            textparams['hidden_dim'] = textparams['hidden_dim']*self.textEncoderModel.textEncoder.numDirs*self.textEncoderModel.seq_dim\n",
    "        \n",
    "        if pretrain:\n",
    "            self.textEncoderModel.trainModel()\n",
    "            self.textEncoderModel.textEncoder.load_state_dict(self.textEncoderModel.optimalParams)\n",
    "            \n",
    "        if treeEncoderType == 'standard':\n",
    "            self.treeEncoderModel = treeEncoder(treeparams['cuda'],treeparams['in_dim'],treeparams['mem_dim'],treeparams['userVects'],treeparams['labels'],treeparams['labelMap'],treeparams['criterion'],device)\n",
    "        if treeEncoderType == 'decay':\n",
    "            self.treeEncoderModel = decayTreeEncoder(treeparams['cuda'],treeparams['in_dim'],treeparams['mem_dim'],treeparams['userVects'],treeparams['labels'],treeparams['labelMap'],treeparams['criterion'],device)\n",
    "        \n",
    "        mem_dim = treeparams['mem_dim'] + textparams['hidden_dim']\n",
    "        \n",
    "        self.fc = nn.Linear(mem_dim,4)    \n",
    "            \n",
    "    def forward(self,tree,text):\n",
    "        treeVec = self.treeEncoderModel(tree)\n",
    "        treeVec = treeVec[0][1].reshape(-1)\n",
    "        \n",
    "        self.textEncoderModel.textEncoder = self.textEncoderModel.textEncoder.to('cpu')\n",
    "        textVec = self.textEncoderModel(text)\n",
    "        textVec = textVec.reshape(-1)\n",
    "#         print(treeVec.shape)\n",
    "#         print(textVec.shape)\n",
    "        combVec =  torch.cat((treeVec,textVec))\n",
    "#         combVec = textVec\n",
    "        out = self.fc(combVec)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainModel(model,modelname):\n",
    "    optimizer = torch.optim.Adagrad(model.parameters(),0.01)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    maxAcc = 0\n",
    "    count = 0\n",
    "    netloss = 0\n",
    "    \n",
    "    for i in range(10):\n",
    "        for treeSet, text in tqdm_notebook(x_train):\n",
    "            tree = treeSet[-1]\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            pred = model(tree.root,text)\n",
    "            \n",
    "            label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "            loss = criterion(pred.reshape(1,4),label)\n",
    "#             print(loss)\n",
    "            netloss += loss\n",
    "    \n",
    "            if count % 20 == 0:\n",
    "#                 print('opt')\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "        \n",
    "        preds = []\n",
    "        labels = []\n",
    "\n",
    "        allLabels = []\n",
    "        allPreds = []\n",
    "        \n",
    "        for valSet, text in tqdm_notebook(x_test):\n",
    "            finalTree = valSet[-1]\n",
    "\n",
    "            predicted = model(finalTree.root,text)\n",
    "            preds.append(predicted)\n",
    "    #         print(predicted)\n",
    "            predicted =  torch.softmax(predicted,0)\n",
    "            predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "            labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "            allLabels.append(labelMap[finalTree.root.label])\n",
    "            allPreds.append(predicted)\n",
    "\n",
    "        predTensor = torch.stack(preds)\n",
    "        labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "        print(allLabels,allPreds)\n",
    "\n",
    "        loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "        cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "        cr['loss'] = loss.item()\n",
    "        cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "\n",
    "        if cr['Acc'] > maxAcc:\n",
    "            maxAcc = cr['Acc']\n",
    "            torch.save({'state_dict': model.state_dict()}, './jointlyTrainedResults/'+modelname+'.pth')\n",
    "\n",
    "        print('loss: ',cr['loss'])\n",
    "        print(cr['Acc'])\n",
    "\n",
    "        with open('./jointlyTrainedResults/'+modelname+'json', 'a') as fp:\n",
    "            json.dump(cr, fp)\n",
    "            fp.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 99/99 [00:00<00:00, 80316.46it/s]\n",
      "\n",
      "  0%|          | 0/200 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 1/200 [00:00<00:24,  8.12it/s]\u001b[A\n",
      "  2%|▏         | 3/200 [00:00<00:21,  8.96it/s]\u001b[A\n",
      "  2%|▏         | 4/200 [00:00<00:21,  8.99it/s]\u001b[A\n",
      "  3%|▎         | 6/200 [00:00<00:19, 10.09it/s]\u001b[A\n",
      "  4%|▍         | 8/200 [00:00<00:17, 11.15it/s]\u001b[A\n",
      "  5%|▌         | 10/200 [00:00<00:15, 12.20it/s]\u001b[A\n",
      "  6%|▌         | 12/200 [00:00<00:15, 12.08it/s]\u001b[A\n",
      "  7%|▋         | 14/200 [00:01<00:18,  9.92it/s]\u001b[A\n",
      "  8%|▊         | 16/200 [00:01<00:29,  6.20it/s]\u001b[A\n",
      "  9%|▉         | 18/200 [00:02<00:24,  7.52it/s]\u001b[A\n",
      " 10%|█         | 20/200 [00:02<00:27,  6.64it/s]\u001b[A\n",
      " 11%|█         | 22/200 [00:02<00:22,  7.98it/s]\u001b[A\n",
      " 12%|█▏        | 24/200 [00:02<00:18,  9.30it/s]\u001b[A\n",
      " 13%|█▎        | 26/200 [00:02<00:16, 10.39it/s]\u001b[A\n",
      " 14%|█▍        | 28/200 [00:02<00:15, 11.36it/s]\u001b[A\n",
      " 15%|█▌        | 30/200 [00:03<00:14, 12.02it/s]\u001b[A\n",
      " 16%|█▌        | 32/200 [00:03<00:13, 12.47it/s]\u001b[A\n",
      " 17%|█▋        | 34/200 [00:03<00:12, 12.99it/s]\u001b[A\n",
      " 18%|█▊        | 36/200 [00:03<00:12, 12.94it/s]\u001b[A\n",
      " 19%|█▉        | 38/200 [00:03<00:14, 11.50it/s]\u001b[A\n",
      " 20%|██        | 40/200 [00:03<00:14, 11.31it/s]\u001b[A\n",
      " 21%|██        | 42/200 [00:04<00:21,  7.43it/s]\u001b[A\n",
      " 22%|██▏       | 43/200 [00:04<00:33,  4.71it/s]\u001b[A\n",
      " 22%|██▏       | 44/200 [00:04<00:29,  5.31it/s]\u001b[A\n",
      " 22%|██▎       | 45/200 [00:05<00:28,  5.35it/s]\u001b[A\n",
      " 23%|██▎       | 46/200 [00:05<00:26,  5.77it/s]\u001b[A\n",
      " 24%|██▍       | 48/200 [00:05<00:23,  6.42it/s]\u001b[A\n",
      " 24%|██▍       | 49/200 [00:05<00:21,  7.01it/s]\u001b[A\n",
      " 25%|██▌       | 50/200 [00:05<00:21,  7.09it/s]\u001b[A\n",
      " 26%|██▌       | 51/200 [00:05<00:19,  7.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.36 recall 0.35833333333333334 prec: 0.3651785714285714 f1: 0.3476190476190476\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.20      0.20      0.20         5\n",
      "           1       0.38      0.50      0.43         6\n",
      "           2       0.29      0.40      0.33         5\n",
      "           3       0.60      0.33      0.43         9\n",
      "\n",
      "   micro avg       0.36      0.36      0.36        25\n",
      "   macro avg       0.37      0.36      0.35        25\n",
      "weighted avg       0.40      0.36      0.36        25\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 26%|██▌       | 52/200 [00:05<00:19,  7.45it/s]\u001b[A\n",
      " 27%|██▋       | 54/200 [00:06<00:17,  8.54it/s]\u001b[A\n",
      " 28%|██▊       | 56/200 [00:06<00:15,  9.05it/s]\u001b[A\n",
      " 28%|██▊       | 57/200 [00:06<00:16,  8.86it/s]\u001b[A\n",
      " 29%|██▉       | 58/200 [00:06<00:17,  8.33it/s]\u001b[A\n",
      " 30%|██▉       | 59/200 [00:06<00:17,  8.01it/s]\u001b[A\n",
      " 30%|███       | 60/200 [00:06<00:16,  8.37it/s]\u001b[A\n",
      " 30%|███       | 61/200 [00:07<00:22,  6.26it/s]\u001b[A\n",
      " 32%|███▏      | 63/200 [00:07<00:19,  7.13it/s]\u001b[A\n",
      " 32%|███▏      | 64/200 [00:07<00:17,  7.59it/s]\u001b[A\n",
      " 33%|███▎      | 66/200 [00:07<00:17,  7.88it/s]\u001b[A\n",
      " 34%|███▎      | 67/200 [00:07<00:16,  7.93it/s]\u001b[A\n",
      " 34%|███▍      | 68/200 [00:07<00:18,  6.97it/s]\u001b[A\n",
      " 35%|███▌      | 70/200 [00:08<00:15,  8.16it/s]\u001b[A\n",
      " 36%|███▌      | 72/200 [00:08<00:15,  8.41it/s]\u001b[A\n",
      " 36%|███▋      | 73/200 [00:08<00:15,  8.15it/s]\u001b[A\n",
      " 38%|███▊      | 75/200 [00:08<00:14,  8.59it/s]\u001b[A\n",
      " 38%|███▊      | 76/200 [00:08<00:19,  6.47it/s]\u001b[A\n",
      " 39%|███▉      | 78/200 [00:09<00:15,  7.68it/s]\u001b[A\n",
      " 40%|████      | 80/200 [00:09<00:14,  8.43it/s]\u001b[A\n",
      " 41%|████      | 82/200 [00:09<00:12,  9.11it/s]\u001b[A\n",
      " 42%|████▏     | 84/200 [00:09<00:14,  8.17it/s]\u001b[A\n",
      " 43%|████▎     | 86/200 [00:09<00:12,  9.16it/s]\u001b[A\n",
      " 44%|████▍     | 88/200 [00:10<00:14,  7.88it/s]\u001b[A\n",
      " 44%|████▍     | 89/200 [00:10<00:14,  7.79it/s]\u001b[A\n",
      " 46%|████▌     | 91/200 [00:10<00:12,  9.04it/s]\u001b[A\n",
      " 46%|████▋     | 93/200 [00:10<00:12,  8.73it/s]\u001b[A\n",
      " 48%|████▊     | 95/200 [00:10<00:11,  9.10it/s]\u001b[A\n",
      " 48%|████▊     | 96/200 [00:11<00:12,  8.34it/s]\u001b[A\n",
      " 48%|████▊     | 97/200 [00:11<00:12,  8.31it/s]\u001b[A\n",
      " 49%|████▉     | 98/200 [00:11<00:12,  8.31it/s]\u001b[A\n",
      " 50%|████▉     | 99/200 [00:11<00:14,  7.09it/s]\u001b[A\n",
      " 50%|█████     | 101/200 [00:11<00:12,  8.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.36 recall 0.35833333333333334 prec: 0.3651785714285714 f1: 0.3476190476190476\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.20      0.20      0.20         5\n",
      "           1       0.38      0.50      0.43         6\n",
      "           2       0.29      0.40      0.33         5\n",
      "           3       0.60      0.33      0.43         9\n",
      "\n",
      "   micro avg       0.36      0.36      0.36        25\n",
      "   macro avg       0.37      0.36      0.35        25\n",
      "weighted avg       0.40      0.36      0.36        25\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 51%|█████     | 102/200 [00:11<00:12,  7.97it/s]\u001b[A\n",
      " 52%|█████▏    | 103/200 [00:11<00:11,  8.42it/s]\u001b[A\n",
      " 52%|█████▏    | 104/200 [00:11<00:11,  8.68it/s]\u001b[A\n",
      " 52%|█████▎    | 105/200 [00:12<00:11,  8.49it/s]\u001b[A\n",
      " 53%|█████▎    | 106/200 [00:12<00:12,  7.53it/s]\u001b[A\n",
      " 54%|█████▎    | 107/200 [00:12<00:13,  6.79it/s]\u001b[A\n",
      " 54%|█████▍    | 108/200 [00:12<00:13,  6.83it/s]\u001b[A\n",
      " 55%|█████▍    | 109/200 [00:12<00:12,  7.32it/s]\u001b[A\n",
      " 55%|█████▌    | 110/200 [00:12<00:11,  7.51it/s]\u001b[A\n",
      " 56%|█████▌    | 111/200 [00:12<00:11,  7.84it/s]\u001b[A\n",
      " 56%|█████▌    | 112/200 [00:13<00:10,  8.09it/s]\u001b[A\n",
      " 56%|█████▋    | 113/200 [00:13<00:11,  7.66it/s]\u001b[A\n",
      " 57%|█████▊    | 115/200 [00:13<00:09,  8.81it/s]\u001b[A\n",
      " 58%|█████▊    | 117/200 [00:13<00:08, 10.01it/s]\u001b[A\n",
      " 60%|█████▉    | 119/200 [00:13<00:08,  9.64it/s]\u001b[A\n",
      " 60%|██████    | 121/200 [00:13<00:07,  9.89it/s]\u001b[A\n",
      " 62%|██████▏   | 123/200 [00:14<00:11,  6.98it/s]\u001b[A\n",
      " 62%|██████▎   | 125/200 [00:14<00:09,  7.81it/s]\u001b[A\n",
      " 63%|██████▎   | 126/200 [00:14<00:09,  8.17it/s]\u001b[A\n",
      " 64%|██████▎   | 127/200 [00:14<00:09,  7.85it/s]\u001b[A\n",
      " 64%|██████▍   | 128/200 [00:14<00:08,  8.24it/s]\u001b[A\n",
      " 64%|██████▍   | 129/200 [00:15<00:09,  7.66it/s]\u001b[A\n",
      " 66%|██████▌   | 131/200 [00:15<00:07,  8.78it/s]\u001b[A\n",
      " 66%|██████▋   | 133/200 [00:15<00:07,  9.26it/s]\u001b[A\n",
      " 67%|██████▋   | 134/200 [00:15<00:07,  9.40it/s]\u001b[A\n",
      " 68%|██████▊   | 135/200 [00:15<00:08,  7.35it/s]\u001b[A\n",
      " 68%|██████▊   | 136/200 [00:15<00:09,  6.58it/s]\u001b[A\n",
      " 69%|██████▉   | 138/200 [00:16<00:08,  7.72it/s]\u001b[A\n",
      " 70%|██████▉   | 139/200 [00:16<00:08,  7.40it/s]\u001b[A\n",
      " 70%|███████   | 141/200 [00:16<00:07,  8.24it/s]\u001b[A\n",
      " 72%|███████▏  | 143/200 [00:16<00:06,  8.90it/s]\u001b[A\n",
      " 72%|███████▎  | 145/200 [00:16<00:05,  9.32it/s]\u001b[A\n",
      " 73%|███████▎  | 146/200 [00:16<00:05,  9.13it/s]\u001b[A\n",
      " 74%|███████▎  | 147/200 [00:17<00:06,  7.93it/s]\u001b[A\n",
      " 74%|███████▍  | 148/200 [00:17<00:07,  7.42it/s]\u001b[A\n",
      " 75%|███████▌  | 150/200 [00:17<00:06,  7.87it/s]\u001b[A\n",
      " 76%|███████▌  | 151/200 [00:17<00:06,  7.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.36 recall 0.35833333333333334 prec: 0.3651785714285714 f1: 0.3476190476190476\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.20      0.20      0.20         5\n",
      "           1       0.38      0.50      0.43         6\n",
      "           2       0.29      0.40      0.33         5\n",
      "           3       0.60      0.33      0.43         9\n",
      "\n",
      "   micro avg       0.36      0.36      0.36        25\n",
      "   macro avg       0.37      0.36      0.35        25\n",
      "weighted avg       0.40      0.36      0.36        25\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 76%|███████▌  | 152/200 [00:17<00:05,  8.09it/s]\u001b[A\n",
      " 76%|███████▋  | 153/200 [00:17<00:05,  8.39it/s]\u001b[A\n",
      " 77%|███████▋  | 154/200 [00:17<00:06,  7.22it/s]\u001b[A\n",
      " 78%|███████▊  | 156/200 [00:18<00:05,  8.44it/s]\u001b[A\n",
      " 79%|███████▉  | 158/200 [00:18<00:04,  9.28it/s]\u001b[A\n",
      " 80%|████████  | 160/200 [00:18<00:04,  9.11it/s]\u001b[A\n",
      " 80%|████████  | 161/200 [00:18<00:04,  9.01it/s]\u001b[A\n",
      " 81%|████████  | 162/200 [00:18<00:04,  9.26it/s]\u001b[A\n",
      " 82%|████████▏ | 163/200 [00:18<00:05,  6.74it/s]\u001b[A\n",
      " 82%|████████▏ | 164/200 [00:19<00:05,  6.13it/s]\u001b[A\n",
      " 83%|████████▎ | 166/200 [00:19<00:04,  7.40it/s]\u001b[A\n",
      " 84%|████████▍ | 168/200 [00:19<00:03,  8.38it/s]\u001b[A\n",
      " 85%|████████▌ | 170/200 [00:19<00:03,  9.53it/s]\u001b[A\n",
      " 86%|████████▌ | 172/200 [00:19<00:02, 10.38it/s]\u001b[A\n",
      " 87%|████████▋ | 174/200 [00:19<00:02, 10.54it/s]\u001b[A\n",
      " 88%|████████▊ | 176/200 [00:20<00:02,  8.69it/s]\u001b[A\n",
      " 88%|████████▊ | 177/200 [00:20<00:03,  7.03it/s]\u001b[A\n",
      " 89%|████████▉ | 178/200 [00:20<00:03,  6.30it/s]\u001b[A\n",
      " 90%|████████▉ | 179/200 [00:20<00:04,  4.95it/s]\u001b[A\n",
      " 90%|█████████ | 181/200 [00:21<00:03,  5.67it/s]\u001b[A\n",
      " 91%|█████████ | 182/200 [00:21<00:02,  6.44it/s]\u001b[A\n",
      " 92%|█████████▏| 183/200 [00:21<00:02,  7.16it/s]\u001b[A\n",
      " 92%|█████████▏| 184/200 [00:21<00:02,  7.55it/s]\u001b[A\n",
      " 92%|█████████▎| 185/200 [00:21<00:01,  7.63it/s]\u001b[A\n",
      " 94%|█████████▎| 187/200 [00:21<00:01,  8.71it/s]\u001b[A\n",
      " 94%|█████████▍| 189/200 [00:21<00:01,  9.85it/s]\u001b[A\n",
      " 96%|█████████▌| 191/200 [00:22<00:00, 10.79it/s]\u001b[A\n",
      " 96%|█████████▋| 193/200 [00:22<00:00, 11.45it/s]\u001b[A\n",
      " 98%|█████████▊| 195/200 [00:22<00:00, 11.83it/s]\u001b[A\n",
      " 98%|█████████▊| 197/200 [00:22<00:00, 11.40it/s]\u001b[A\n",
      "100%|██████████| 200/200 [00:22<00:00,  8.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.36 recall 0.35833333333333334 prec: 0.3651785714285714 f1: 0.3476190476190476\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.20      0.20      0.20         5\n",
      "           1       0.38      0.50      0.43         6\n",
      "           2       0.29      0.40      0.33         5\n",
      "           3       0.60      0.33      0.43         9\n",
      "\n",
      "   micro avg       0.36      0.36      0.36        25\n",
      "   macro avg       0.37      0.36      0.35        25\n",
      "weighted avg       0.40      0.36      0.36        25\n",
      "\n",
      "attn_standard-tree_pretrain-True_lstmbidir-True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5ffd482e25140a59275c58825e92f13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikhil.pinnaparaju/anaconda3/envs/fakenews/lib/python3.7/site-packages/ipykernel_launcher.py:17: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-5f74015d2d06>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     39\u001b[0m                         \u001b[0mmodelname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtextType\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mtreeType\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'-tree_pretrain-'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrainType\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mrnnType\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'bidir-'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbidirType\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m                         \u001b[0mtrainModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-25-ea7996fe9956>\u001b[0m in \u001b[0;36mtrainModel\u001b[0;34m(model, modelname)\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabelMap\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtreeSet\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fakenews/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-24-d89f922676a5>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tree, text)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtree\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0mtreeVec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtreeEncoderModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtree\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m         \u001b[0mtreeVec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtreeVec\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fakenews/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-610cca0e0d71>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, node)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_children\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildrenList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mchild_loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0mchild_c\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_h\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetChildStates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-610cca0e0d71>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, node)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_children\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildrenList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mchild_loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0mchild_c\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_h\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetChildStates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-610cca0e0d71>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, node)\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mchild_loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0mchild_c\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_h\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetChildStates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnodeForward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muserVects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchild_c\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchild_h\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputModule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-610cca0e0d71>\u001b[0m in \u001b[0;36mnodeForward\u001b[0;34m(self, inputs, child_c, child_h)\u001b[0m\n\u001b[1;32m     68\u001b[0m         f = torch.sigmoid(\n\u001b[1;32m     69\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchild_h\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchild_h\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         )\n\u001b[1;32m     72\u001b[0m         \u001b[0mfc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_c\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "treeparams = {\n",
    "    'cuda': torch.cuda.is_available(),\n",
    "    'in_dim':8,\n",
    "    'mem_dim':100,\n",
    "    'userVects':userVects,\n",
    "    'labels':twitter15_labels,\n",
    "    'labelMap':labelMap,\n",
    "    'criterion':nn.CrossEntropyLoss()\n",
    "}\n",
    "\n",
    "textparams = {\n",
    "    'embedding_dim':256,\n",
    "    'hidden_dim': 50,\n",
    "    'output_dim':4,\n",
    "    'num_layers':1,\n",
    "    'bidir':True,\n",
    "    'rnnType':'gru'\n",
    "}\n",
    "\n",
    "treeTypes = ['standard','decay']\n",
    "textTypes = ['attn']\n",
    "attnTypes = ['self','dot']\n",
    "pretrainTypes = [True,False]\n",
    "rnnTypes = ['lstm','gru']\n",
    "bidirTypes = [True,False]\n",
    "\n",
    "for textType in textTypes:\n",
    "    for treeType in treeTypes:\n",
    "        for pretrainType in pretrainTypes:\n",
    "            for rnnType in rnnTypes:\n",
    "                for bidirType in bidirTypes:\n",
    "                    for attnType in attnTypes:\n",
    "                        textparams['rnnType'] = rnnType\n",
    "                        textparams['bidirType'] = bidirType\n",
    "                        textparams['attnType'] = attnType\n",
    "\n",
    "                        model = jointModel(treeType,textType,pretrainType,textparams,treeparams,X_text,y,device)\n",
    "                        model = model.to(device)\n",
    "                        modelname = textType+'_'+treeType+'-tree_pretrain-'+str(pretrainType)+'_'+rnnType+'bidir-'+str(bidirType)\n",
    "                        print(modelname)\n",
    "                        trainModel(model,modelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 10:10:34.379012 139830510675712 tokenization_utils.py:374] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/nikhil.pinnaparaju/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "I0107 10:10:35.349486 139830510675712 tokenization_utils.py:374] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/nikhil.pinnaparaju/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "I0107 10:10:36.743212 139830510675712 configuration_utils.py:151] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /home/nikhil.pinnaparaju/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.bf3b9ea126d8c0001ee8a1e8b92229871d06d36d8808208cc2449280da87785c\n",
      "I0107 10:10:36.745564 139830510675712 configuration_utils.py:168] Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 4,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "I0107 10:10:37.689841 139830510675712 modeling_utils.py:337] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /home/nikhil.pinnaparaju/.cache/torch/transformers/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([13, 4])\n",
      "acc: 0.44878048780487806 recall 0.45047138047138047 prec: 0.4910589410589411 f1: 0.43515065938849146\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.35      0.50      0.41        46\n",
      "           1       0.43      0.70      0.54        54\n",
      "           2       0.68      0.38      0.49        50\n",
      "           3       0.50      0.22      0.30        55\n",
      "\n",
      "   micro avg       0.45      0.45      0.45       205\n",
      "   macro avg       0.49      0.45      0.44       205\n",
      "weighted avg       0.49      0.45      0.43       205\n",
      "\n",
      "\n",
      "\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([13, 4])\n",
      "acc: 0.551219512195122 recall 0.5476804274630361 prec: 0.5398030549346339 f1: 0.5337583088155171\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.42      0.35      0.38        46\n",
      "           1       0.68      0.76      0.72        54\n",
      "           2       0.51      0.72      0.60        50\n",
      "           3       0.54      0.36      0.43        55\n",
      "\n",
      "   micro avg       0.55      0.55      0.55       205\n",
      "   macro avg       0.54      0.55      0.53       205\n",
      "weighted avg       0.54      0.55      0.54       205\n",
      "\n",
      "\n",
      "\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([13, 4])\n",
      "acc: 0.6536585365853659 recall 0.6467179036744254 prec: 0.6742555159893869 f1: 0.6357609315899145\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.37      0.49        46\n",
      "           1       0.74      0.85      0.79        54\n",
      "           2       0.53      0.82      0.65        50\n",
      "           3       0.71      0.55      0.62        55\n",
      "\n",
      "   micro avg       0.65      0.65      0.65       205\n",
      "   macro avg       0.67      0.65      0.64       205\n",
      "weighted avg       0.68      0.65      0.64       205\n",
      "\n",
      "\n",
      "\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([32, 4])\n",
      "torch.Size([13, 4])\n",
      "acc: 0.6829268292682927 recall 0.6798653198653197 prec: 0.7097582972582972 f1: 0.6745905992741436\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.50      0.58        46\n",
      "           1       0.84      0.87      0.85        54\n",
      "           2       0.53      0.84      0.65        50\n",
      "           3       0.78      0.51      0.62        55\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       205\n",
      "   macro avg       0.71      0.68      0.67       205\n",
      "weighted avg       0.71      0.68      0.68       205\n",
      "\n",
      "\n",
      "\n",
      "bert_standard-tree_pretrain-True\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25391ccd51414e6cb2892f40d1e5a59d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikhil.pinnaparaju/anaconda3/envs/fakenews/lib/python3.7/site-packages/ipykernel_launcher.py:17: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a404257bc37e437fbdafb99e840cae64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3, 3, 2, 3, 3, 3, 0, 3, 1, 2, 3, 0, 0, 3, 1, 0, 3, 1, 1, 0, 0, 0, 3, 3, 1, 3, 3, 2, 1, 2, 1, 1, 2, 3, 2, 2, 0, 1, 1, 0, 0, 3, 1, 3, 0, 3, 3, 2, 3, 2, 0, 0, 2, 3, 3, 0, 1, 3, 2, 2, 2, 0, 2, 2, 0, 0, 3, 2, 2, 3, 0, 3, 1, 0, 0, 0, 1, 1, 0, 2, 1, 1, 0, 1, 3, 2, 2, 2, 1, 3, 2, 1, 2, 1, 0, 1, 3, 3, 2, 0, 1, 3, 2, 3, 1, 3, 0, 1, 2, 0, 0, 2, 3, 3, 0, 0, 2, 2, 2, 3, 2, 0, 3, 0, 0, 2, 3, 0, 1, 0, 0, 2, 1, 1, 1, 2, 2, 0, 1, 1, 3, 0, 0, 2, 1, 1, 0, 1, 0, 0, 1, 2, 2, 2, 3, 3, 2, 3, 1, 3, 2, 1, 3, 3, 2, 3, 2, 1, 0, 0, 3, 0, 2, 0, 3, 1, 2, 3, 3, 2, 3, 3, 3, 3, 2, 1, 2, 3, 1, 3, 0, 0, 0, 1, 1, 1, 0, 0, 2, 1, 3, 3, 3, 0, 2] [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n",
      "loss:  1.5467243194580078\n",
      "0.24390243902439024\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bcd61fa069d46ccafcddd3273d932db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dccd6cfafe844ddebbdf2865b6a7b411",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "treeparams = {\n",
    "    'cuda': torch.cuda.is_available(),\n",
    "    'in_dim':8,\n",
    "    'mem_dim':100,\n",
    "    'userVects':userVects,\n",
    "    'labels':twitter15_labels,\n",
    "    'labelMap':labelMap,\n",
    "    'criterion':nn.CrossEntropyLoss()\n",
    "}\n",
    "\n",
    "textparams = {\n",
    "    'embedding_dim':256,\n",
    "    'hidden_dim': 50,\n",
    "    'output_dim':4,\n",
    "    'num_layers':1,\n",
    "    'bidir':True,\n",
    "    'rnnType':'gru'\n",
    "}\n",
    "\n",
    "treeTypes = ['standard','decay']\n",
    "textTypes = ['bert']\n",
    "pretrainTypes = [True,False]\n",
    "\n",
    "treeTypes = ['standard','decay']\n",
    "textTypes = ['bert']\n",
    "pretrainTypes = [True]\n",
    "\n",
    "for textType in textTypes:\n",
    "    for treeType in treeTypes:\n",
    "        for pretrainType in pretrainTypes:\n",
    "            model = jointModel(treeType,textType,pretrainType,textparams,treeparams,X_text,y,device)\n",
    "            model = model.to(device)\n",
    "            modelname = textType+'_'+treeType+'-tree_pretrain-'+str(pretrainType)\n",
    "            print(modelname)\n",
    "            trainModel(model,modelname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "treeparams = {\n",
    "    'cuda': torch.cuda.is_available(),\n",
    "    'in_dim':8,\n",
    "    'mem_dim':100,\n",
    "    'userVects':userVects,\n",
    "    'labels':twitter15_labels,\n",
    "    'labelMap':labelMap,\n",
    "    'criterion':nn.CrossEntropyLoss()\n",
    "}\n",
    "\n",
    "textparams = {\n",
    "    'embedding_dim':256,\n",
    "    'hidden_dim': 50,\n",
    "    'output_dim':4,\n",
    "    'num_layers':1,\n",
    "    'bidir':True,\n",
    "    'rnnType':'gru'\n",
    "}\n",
    "\n",
    "treeTypes = ['decay']\n",
    "textTypes = ['rnn']\n",
    "pretrainTypes = [True,False]\n",
    "bidirTypes = [True,False]\n",
    "rnnTypes = ['lstm','gru']\n",
    "\n",
    "for textType in textTypes:\n",
    "    for treeType in treeTypes:\n",
    "        for pretrainType in pretrainTypes:\n",
    "            for rnnType in rnnTypes:\n",
    "                for bidirType in bidirTypes:\n",
    "                    textparams['rnnType'] = rnnType\n",
    "                    textparams['bidirType'] = bidirType\n",
    "                    \n",
    "                    model = jointModel(treeType,textType,pretrainType,textparams,treeparams,X_text,y,device)\n",
    "                    model = model.to(device)\n",
    "                    modelname = textType+'_'+treeType+'-tree_pretrain-'+str(pretrainType)+'_'+rnnType+'bidir-'+str(bidirType)\n",
    "                    print(modelname)\n",
    "                    trainModel(model,modelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fakenews",
   "language": "python",
   "name": "fakenews"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
